# -*- coding: utf-8 -*-
'''
| @author: olivia.dou
| Created on: 2020/1/9 14:57
| desc: 和workflow相关的功能
'''

import logging,traceback,time,re,os
import pandas as pd
import base.globalvars as glo
from util.db_util import execute_query
from util.date_util import timestamp_to_str
from proj_spec.DAS.model import get_model_auto_task_id,get_model_element_ids,get_model_info
from proj_spec.DAS.element import get_element_name,get_element_id #get_published_elements_count
from proj_spec.DAS.std import validate_std_data_trace
from proj_spec.DAS.coa import validate_template_data_track
from proj_spec.DAS.extractdata import get_extract_data_info,reset_data
from .stdqa import get_std_qc_element_result

STEP_FIELDS_MAP = {
                   0:{"data_status":"extract","interface":"extractStatus","cost":10,"timeout":20,"stage":"Extract","task_item":"extract"},
                   1:{"data_status":"struct","interface":"structStatus","cost":10,"timeout":20,"stage":"Struct","task_item":"struct"},
                   2:{"data_status":"qa","interface":"structQaStatus","cost":10,"timeout":20,"stage":"StructQA","task_item":"struct_qa"},
                   3:{"data_status":"std","interface":"stdStatus","cost":10,"timeout":20,"stage":"Std","task_item":"std"},
                   4:{"data_status":"std_qa","interface":"stdQaStatus","cost":10,"timeout":20,"stage":"StdQA","task_item":"std_qa"},
                   5:{"data_status":"","interface":"tlpCalcStatus","cost":10,"timeout":20,"stage":"TplMappingCalc","task_item":"tlp_calc"},
                   6:{"data_status":"","interface":"tlpQaStatus","cost":10,"timeout":20,"stage":"TplMappingQA","task_item":"tlp_qa"},
                   7:{"data_status":"","interface":"tlpAdjustStatus","cost":10,"timeout":20,"stage":"TplMappingCalc","task_item":"tlp_adjust"}
                   }

ONE_CLICK_EXTRACT_STEPS = [0,1,3,4,5,7,6]
PHASE3_STEPS = [5,7,6]

RETRY_COUNT = 10

#PUBLISHED_ELEMENTS_COUNT = get_published_elements_count()

def validate_file_operation_audit(file_id, model_id, step, start_time_str):
    """验证file_operation_audit表记录了提取过程执行结果，且提取、结构化、标准化、模板计算步骤无失败记录

    :param file_id: 文件id
    :param model_id: 模型id
    :param step: 步骤
    :param start_time_str: 开始时间字符串，如2019-10-10 21:03:30

    :return: True 验证通过, False 验证失败
    """
    prefix = STEP_FIELDS_MAP[step]['file_audit']
    if prefix == "":  # db字段为空的不检查
        return True

    sql = "SELECT start_time,end_time,param_map,status,message FROM t_das_file_operation_audit WHERE file_id=%s and step='%s' " \
           % (file_id, prefix)

    res = execute_query(sql)
    if len(res) == 0:
        logging.error("audit表未新增%s记录：模型id: %s,文件id:%s" % (prefix, model_id, file_id))
        return False  # audit表应有新增记录
    dataseries = pd.Series(res[0], index=['start_time', 'end_time', 'param_map','status','message'])

    # 检查时间、状态字段
    # if dataseries['start_time_str'].timestamp()<start_time_str: #数据库中的时间本身做了向下取整，可能会小于提取过程开始时间
    if dataseries['start_time'] is None or dataseries['start_time'] == '' or dataseries['start_time'] < start_time_str : #转成GMT时间
        logging.error("file audit表未生成新%s audit记录：模型id: %s,文件id:%s" % (prefix, model_id, file_id))
        return False
    if prefix in ['Extract', 'Struct','Std','TplMappingCalc'] and dataseries['status'] == 1:
    #if dataseries['status'] == 1:
        logging.error("file audit表%s失败：模型id: %s,文件id:%s,message:%s" % (prefix, model_id, file_id,dataseries['message'] ))
        return False
    #if 'Tpl'  in prefix '': #mapping为模板、映射关系

    return True



def get_phase1_detail(file_id, model_id, element_id, step, start_time_str, expect_executed=True):
    """获取dashboard_phase1_detail表的状态

    :param file_id: 文件id
    :param model_id: 模型id
    :param element_id: 元素id
    :param step: 步骤 0：提取 1：结构化 2：结构化质检（已废弃） 3：标准化 4：标准化质检 5：模板计算 6：模板质检
    :param start_time_str: 起始时间字符串，如2019-10-10 21:03:30

    :return: 对应的状态，0 成功 1 失败 2 警告
    """

    stage = STEP_FIELDS_MAP[step]['stage']

    sql = """select status,message,upt from t_das_dashboard_phase1_detail where file_id=%s and model_id=%s and element_id=%s
     and stage='%s' and upt>='%s' order by upt desc""" %(file_id,model_id,element_id,stage,start_time_str)

    count=RETRY_COUNT
    while count>0:
        try:
            # cur_time = timestamp_to_str(time.time())
            res = execute_query(sql,result_type="dict")
            if not expect_executed:
                assert len(res)==0
                return None
            else:
                assert len(res)==1
                result = res[0]['status']
                # assert result == 0, "phase1不成功：file_id %s model_id %s element_id %s stage %s, status:%s, message：%s" % (
                #     file_id, model_id, element_id, stage, record['status'], record['message'])  # 为尽可能多地发现和排除问题，这里把状态为警告也断言失败
                return result
        except Exception as e:
            if count==1:
                logging.error("%s: %s不成功, file id %s, model_id %s, element_id %s, start_time %s, 预期执行 %s"
                              %(e,stage,file_id,model_id,element_id,start_time_str,expect_executed))
                raise e
            else:
                logging.info("sleeping 3 seconds...")
                time.sleep(3)
            count-=1


def get_phase2_detail(file_id, step, qc_group, start_time_str, expect_executed=True):
    """获取dashboard_phase2_detail表的状态信息

    :param file_id: 文件id
    :param step: 步骤 0：提取 1：结构化 2：结构化质检（已废弃） 3：标准化 4：标准化质检 5：模板计算 6：模板质检
    :param qc_group: 质检分组id
    :param start_time_str: 起始时间字符串，如2019-10-10 21:03:30
    :param expect_executed: True: 验证该步骤被执行 False: 验证该步骤未执行

    :return: 对应的状态，0 成功 1 失败 2 警告
    """

    stage = STEP_FIELDS_MAP[step]['stage']
    sql = """select status,message,upt from t_das_dashboard_phase2_detail where file_id=%s and stage='%s' and qcgroup=%s 
        and upt>='%s' order by upt desc"""%(file_id,stage,qc_group,start_time_str)

    count = RETRY_COUNT
    while count > 0:
        try:
            res = execute_query(sql,result_type="dict")
            if not expect_executed:
                assert len(res)==0, "预期%s不执行，实际执行：%s"%(step, file_id)
                return None
            else:
                assert len(res)==1
                record = res[0]
                #result = res[0]['status']
                assert record['status'] != 1 or record['status'] == 1 and re.match(r"共质检\d+条规则;.+;", record[
                    'message']) is not None, "phase2不成功：file_id %s stage %s, status:%s, message:%s" % (
                file_id, stage, record['status'], record['message'])
                return record['status']
        except Exception as e:
            if count == 1:
                logging.error(e)
                raise e
            else:
                logging.info("sleeping 3 seconds...")
                time.sleep(3)
            count -= 1



def get_phase3_detail(file_id, mapping_id, step, start_time_str, expect_executed=True):
    """获取dashboard_phase3_detail表的状态

    :param file_id: 文件id
    :param mapping_id: 模型id
    :param step: 步骤 0：提取 1：结构化 2：结构化质检（已废弃） 3：标准化 4：标准化质检 5：模板计算 6：模板质检
    :param start_time_str: 起始时间字符串，如2019-10-10 21:03:30

    :return: 对应的状态，0 成功 1 失败 2 警告
    """

    stage = STEP_FIELDS_MAP[step]['stage']
    sql = """select status,message,upt from t_das_dashboard_phase3_detail where file_id=%s and mapping_id=%s and stage='%s' order by upt desc""" \
          % (file_id, mapping_id, stage)
    count = RETRY_COUNT
    cur_time = time.strftime("%Y-%m-%d %H:%M:%S")
    while count > 0:

        try:
            res = execute_query(sql, result_type="dict")

            if not expect_executed:
                assert len(res)==0
                return None
            else:
                assert len(res)==1,"phase3 detail表未更新:当前时间 %s file_id %s mapping_id %s stage %s" % (cur_time,file_id, mapping_id, stage)
                record = res[0]
                #result = res[0]['status']
                assert record['upt'] is not None and record['upt'] != '' and record['upt'] > pd.to_datetime(start_time_str),\
                    "phase3 upt小于start_time: 查询时间 %s, file_id %s mapping_id %s stage %s, start_time_str:%s, upt:%s" % (
                        cur_time, file_id, mapping_id, stage, start_time_str, record['upt'])
                assert record['status'] != 1 if step==5 else True,"phase3不成功：file_id %s mapping_id %s stage %s, status:%s, message:%s"\
                                                                  % (file_id, mapping_id, stage, record['status'], record['message'])
                return record['status']

        except Exception as e:
            if count == 1:
                logging.error(e)
                raise e
            else:
                logging.info("sleeping 3 seconds...")
                time.sleep(3)
            count -= 1


def validate_task_item_update(task_id, file_id, steps, start_time_str, expect_executed=True):
    """验证task_item表的更新

    :param task_id: 任务id
    :param file_id: 文件id
    :param steps: 步骤 0：提取 1：结构化 2：结构化质检（已废弃） 3：标准化 4：标准化质检 5：模板计算 6：模板质检
    :param start_time_str: 起始时间字符串，如2019-10-10 21:03:30
    :param expect_executed: True: 验证该步骤被执行 False: 验证该步骤未执行

    :return: True 验证通过, False 验证失败
    """
    # sql = "SELECT extract_status,struct_status,struct_qa_status,std_status,std_qa_status,tlp_calc_status,tlp_qa_status," \
    #       "upt FROM t_das_auto_task_item WHERE task_id=%s and file_id=%s and upt>='%s'" % (task_id, file_id, start_time_str)
    sql = "SELECT * FROM t_das_auto_task_item WHERE task_id=%s and file_id=%s and upt>='%s'" % (task_id, file_id, start_time_str)
    cur_time = time.time()
    cur_timestr = timestamp_to_str(cur_time)
    res = execute_query(sql,result_type='dict')
    if not expect_executed:
        assert len(res)==0,"预期不执行，实际执行：task id: %s,文件id:%s" % (task_id, file_id)
        return True
    else:
        assert len(res) == 1, "%s task item表记录数不正确，预期1条，实际%s条:task id: %s,文件id:%s" % (start_time_str, len(res), task_id, file_id)
        # dataseries = pd.Series(res[0],
        #                        index=['extract_status', 'struct_status', 'struct_qa_status', 'std_status', 'std_qa_status',
        #                               'tlp_calc_status', 'tlp_qa_status', 'upt'])
        dataseries = pd.Series(res[0])

        # 验证更新, 但继续任务时不验证更新时间，因为数据完全一致时，按照mysql的机制，upt不会更新
        if task_id==-1:
            assert dataseries['upt'] > pd.to_datetime(start_time_str) , "task item表未更新：task id: %s,文件id:%s,当前时间：%s" % (
            task_id,  file_id, cur_timestr)

        # # 提取和结构化应有值
        # assert dataseries['extract_status'] is not None and dataseries['struct_status' is not None], "提取结构化状态字段为空"

        # 验证执行步骤的状态均有值
        for step in steps:
            prefix = STEP_FIELDS_MAP[step]['task_item']
            if prefix == "":  # db字段为空的不检查
                return True


            # Todo: 改为验证task item状态与各元素执行状态一致（start_time之后）
            assert dataseries['%s_status' % prefix] != 0, "task item %s未执行: task id %s,文件id %s,process start time: %s"\
                                                          %(prefix,task_id,file_id,pd.to_datetime(start_time_str))

    return True


def validate_phase1(file_id, model_name, steps, start_time_str, element_name="", expect_executed=True):
    """验证phase1表的更新

    :param file_id: 文件id
    :param model_name: 模型名称
    :param steps: 步骤
    :param start_time_str: 起始时间字符串，如2019-10-10 21:03:30
    :param element_name: 元素名称
    :param expect_executed: True: 验证该步骤被执行 False: 验证该步骤未执行 Todo: 补充phase1验证未执行的逻辑

    :return: True 验证通过, False 验证失败
    """

    model_info = get_model_info(model_name=model_name)
    if model_info['status'] != 4: #非已发布
        logging.error("model %s is not published"%model_name)
        return False
    model_id = model_info['id']

    if element_name=="":
        element_ids = get_model_element_ids(model_id=model_id)
    elif element_name is not None:
        element_id = get_element_id(model_name,element_name)
        element_ids = [element_id]
    else:
        element_ids = [] # element_name is None, 即只验证step 4情况（如一键重述）

    # 获取模型上线时的全自动任务id
    task_id = get_model_auto_task_id(model_id)

    try:
        for step in steps:

            if step in [0, 1, 3]:  # 提取、结构化、标准化
                for element_id in element_ids:
                    if not expect_executed:
                        assert get_phase1_detail(file_id, model_id, element_id, step,
                                                 start_time_str, expect_executed=False) is None, "%s预期不执行，实际执行: 文件%s 模型%s 元素%s" % (
                        STEP_FIELDS_MAP[step]['stage'], file_id, model_id, element_id)
                    else:
                        if is_exempted(file_id,model_id,element_id):
                            continue
                        #assert get_phase1_detail(file_id, model_id, element_id, step, start_time_str) not in [1, -1]
                        assert get_phase1_detail(file_id, model_id, element_id, step, start_time_str) == 0, "%s不成功: 文件%s 模型%s 元素%s" % (STEP_FIELDS_MAP[step]['stage'], file_id, model_id, element_id)
            if step == 3:
                for element_id in element_ids:
                    if is_exempted(file_id, model_id, element_id):  # 模型元素不适用，跳过
                        continue
                    elif get_extract_data_info(model_id=model_id, element_id=element_id, file_id=file_id)[
                        'status'] == 1:
                        reset_data(file_id, model_id, element_id,need_notify=True)
                    else:
                        assert validate_std_data_trace(file_id, model_id, element_id, start_time_str), "标准化数据追踪验证失败：file_id %s,model_id %s,element_id %s" % (
                                        file_id, model_id, element_id)
                # Todo:验证标准化数据表里的数据是新生成的

        assert validate_task_item_update(task_id, file_id, steps, start_time_str, expect_executed) is True, "task item更新验证失败"
        return True
    except Exception as e:
        logging.error(e)
        traceback.print_exc()

        return False


def validate_phase2(file_id, model_name, steps, start_time_str, element_name="", expect_executed=True):
    """验证phase2表的更新

    :param file_id: 文件id
    :param model_name: 模型名称
    :param steps: 步骤
    :param start_time_str: 起始时间字符串，如2019-10-10 21:03:30
    :param element_name: 元素名称
    :param expect_executed: True: 验证该步骤被执行 False: 验证该步骤未执行

    :return: True 验证通过, False 验证失败
    """

    model_info = get_model_info(model_name=model_name)
    if model_info['status'] != 4: #非已发布
        logging.error("model %s is not published"%model_name)
        return False
    model_id = model_info['id']

    # 获取模型上线时的全自动任务id
    task_id = get_model_auto_task_id(model_id)

    try:
        for step in steps:

            if step == 4:  # 标准化质检
                qc_group = get_model_info(model_id=model_id)['qcgroup']
                if expect_executed:
                    assert (get_phase2_detail(file_id, step, qc_group, start_time_str) != -1)
                else:
                    assert (get_phase2_detail(file_id, step, qc_group, start_time_str,expect_executed=False) is None)

        # assert validate_task_item_update(task_id, file_id, steps, start_time_str, expect_executed) is True, "task item更新验证失败"
        return True
    except Exception as e:
        logging.error(e)
        traceback.print_exc()

        return False


def validate_phase3(file_id, steps, start_time_str, skip_trace_validation=False,expect_executed=True):
    """验证phase3表数据

    :param file_id: 文件id
    :param steps: 步骤 0：提取 1：结构化 2：结构化质检（已废弃） 3：标准化 4：标准化质检 5：模板计算 6：模板质检 7:调整计算
    :param start_time_str: 起始时间字符串，如2019-10-10 21:03:30
    skip_trace_validation: 是否跳过追踪验证

    :return: True 验证通过, False 验证失败
    """
    from proj_spec.DAS.mapping import get_fit_mappings, get_template_calc_mappings #不在全局导入，解决循环导入的问题
    from proj_spec.DAS.File import get_file_entity_code, get_file_params_for_template_mapping_crieria
    from proj_spec.DAS.coa import get_coa_review_status,revert_coa_review,adjust_calc

    try:
        for step in steps:
            fit_mapping_ids = get_fit_mappings(file_id)

            whitelist_mapping_ids = glo.get_value("validate_mapping_ids").split(',')
            if whitelist_mapping_ids != ['']:
                whitelist_mapping_ids = [int(id) for id in whitelist_mapping_ids]
            else:
                whitelist_mapping_ids = []

            mapping_ids = list(set(fit_mapping_ids).intersection(set(whitelist_mapping_ids)))
            # 验证mapping_ids与实际调用到的相符
            cur_time_str = timestamp_to_str(time.time())
            calculated_mappings = get_template_calc_mappings(file_id, start_time_str)
            assert set(fit_mapping_ids) == set(
                calculated_mappings), "%s 文件 %s 匹配的映射与模板计算调用的映射不符：匹配的映射 %s,模板计算调用的映射 %s" % (cur_time_str, file_id,
                fit_mapping_ids, calculated_mappings)

            company_code = get_file_entity_code(file_id)
            report_year, report_period, category_id, type_id, tag_ids = get_file_params_for_template_mapping_crieria(
                file_id)
            for mapping_id in mapping_ids:

                # Todo: 调整计算？
                #if step == 5:
                if step in [5, 7]:
                    logging.info("mapping id: %s" % mapping_id)
                    assert get_phase3_detail(file_id, mapping_id, step, start_time_str, expect_executed) not in [1,-1], "模板计算失败:file_id %s mapping_id %s start_time_str %s "%(file_id, mapping_id,start_time_str)

                    if not skip_trace_validation:
                        review_status = get_coa_review_status(report_year,report_period,mapping_id,company_code,file_id)
                        if review_status==1:#审核通过
                            #撤销COA审核
                            revert_coa_review(file_id, mapping_id)
                            #重新计算
                            adjust_calc(file_id, mapping_id)
                            time.sleep(5)
                        assert validate_template_data_track(company_code, report_year, report_period, mapping_id,
                                                        file_id), "模板数据追踪验证失败：company_code %s,report_year %s,report_period %s,mapping_id %s,file_id %s" % (
                        company_code, report_year, report_period, mapping_id, file_id)
                else:
                    assert get_phase3_detail(file_id, mapping_id, step, start_time_str) != -1, "模板质检失败：file_id %s mapping_id %s start_time_str %s"%(file_id, mapping_id,start_time_str)

        return True
    except Exception as e:
        logging.error(e)
        traceback.print_exc()
        return False

def get_exemption_file_list():
    """从文件中读取不适用的文件模型组合，以供跳过提取过程、一键提取

    :return: 文件,模型
    """

    from base.get_config import get_tc_rootdir
    tc_roodir = get_tc_rootdir()
    #data = pd.read_excel(tc_roodir + os.sep + "exemption_file.xlsx", header=[0, 1])
    data = pd.read_excel(tc_roodir + os.sep + "exemption_file.xlsx",header=[0,1], engine="openpyxl") #不加engine, pytest执行会有warning
    data = data.set_index(data.columns[0])
    return data

def is_exempted(file_id,model_id,element_id):
    """判断指定的文件模型元素是否不适用

    :param file_id: 文件id
    :param model_id: 模型id
    :param element_id: 元素id

    :return: True 不适用  False 适用
    """

    model_info = get_model_info(model_id = model_id)
    #model_name = get_model_name(model_id)
    model_name = model_info['model_name']

    element_name = get_element_name(element_id)

    data = get_exemption_file_list()
    try:
        if (model_name,element_name) in data.columns and file_id in data[model_name,element_name].index and data[model_name,element_name][file_id]=='N':
            return True
        else:
            return False
    except Exception as e:
        logging.error(e)
        traceback.print_exc()
        return False


def get_time_cost(stages):
    return sum([STEP_FIELDS_MAP[i]['cost'] for i in stages])


# def validate_phase1_result(file_id, element_id, start_time_str, expected_status):
#     """验证phase1表的更新
#
#     :param file_id: 文件id
#     :param element_id: 元素id
#     :param start_time_str: 起始时间字符串，如2019-10-10 21:03:30
#
#     :return: True 验证通过, False 验证失败
#     """
#
#
#     try:
#         for step in steps:
#
#             if step in [0, 1, 3]:  # 提取、结构化、标准化
#                 for element_id in element_ids:
#                     if not expect_executed:
#                         assert get_phase1_detail(file_id, model_id, element_id, step,
#                                                  start_time_str, expect_executed=False) is None, "%s预期不执行，实际执行: 文件%s 模型%s 元素%s" % (
#                         STEP_FIELDS_MAP[step]['stage'], file_id, model_id, element_id)
#                     else:
#                         if is_exempted(file_id,model_id,element_id):
#                             continue
#                         #assert get_phase1_detail(file_id, model_id, element_id, step, start_time_str) not in [1, -1]
#                         assert get_phase1_detail(file_id, model_id, element_id, step, start_time_str) == 0, "%s不成功: 文件%s 模型%s 元素%s" % (STEP_FIELDS_MAP[step]['stage'], file_id, model_id, element_id)
#             if step == 3:
#                 for element_id in element_ids:
#                     if is_exempted(file_id, model_id, element_id):  # 模型元素不适用，跳过
#                         continue
#                     elif get_extract_data_info(model_id=model_id, element_id=element_id, file_id=file_id)[
#                         'status'] == 1:
#                         reset_data(file_id, model_id, element_id,need_notify=True)
#                     else:
#                         assert validate_std_data_trace(file_id, model_id, element_id, start_time_str), "标准化数据追踪验证失败：file_id %s,model_id %s,element_id %s" % (
#                                         file_id, model_id, element_id)
#                 # Todo:验证标准化数据表里的数据是新生成的
#
#         assert validate_task_item_update(task_id, file_id, steps, start_time_str, expect_executed) is True, "task item更新验证失败"
#         return True
#     except Exception as e:
#         logging.error(e)
#         traceback.print_exc()
#
#         return False

def get_phase1_detail_status(file_id, element_id, step, start_time, timeout=60):
    """

    :param file_id: 文件id
    :param element_id: 元素id
    :param step: 步骤编号 0,1,3,4,5,6,7
    :param start_time: 起始时间，用于查询最新的执行记录时限定最近的时间（避免查到的记录为历史执行记录）
    :param timeout: 超时时间（秒）

    :return: 状态值
    """

    start_time_str = timestamp_to_str(start_time)
    sql = "SELECT status FROM t_das_dashboard_phase1_detail WHERE file_id=%s  and element_id=%s AND stage='%s' and upt>='%s'"\
          %(file_id, element_id, STEP_FIELDS_MAP[step]['stage'],start_time_str)
    res = execute_query(sql)
    while res==[] and time.time()-int(glo.get_value("time_difference",defValue=0)) - start_time < timeout:
        res = execute_query(sql)
        time.sleep(3)

    return res


def get_phase3_detail_status(file_id, step, start_time, timeout=60):
    """

    :param file_id: 文件id
    :param element_id: 元素id
    :param step: 步骤编号 0,1,3,4,5,6,7
    :param start_time: 起始时间，用于查询最新的执行记录时限定最近的时间（避免查到的记录为历史执行记录）
    :param timeout: 超时时间（秒）

    :return: 状态值
    """

    start_time_str = timestamp_to_str(start_time)
    sql = "SELECT status FROM t_das_dashboard_phase3_detail WHERE file_id=%s  AND stage='%s' and upt>='%s'"\
          %(file_id,STEP_FIELDS_MAP[step]['stage'],start_time_str)
    res = execute_query(sql)
    while res==[] and time.time()-int(glo.get_value("time_difference",defValue=0)) - start_time < timeout:
        res = execute_query(sql)
        time.sleep(3)

    return res


def get_phase1_detail_result(file_elements, step, start_time, timeout=60):
    """

    :param file_elements:
    :param start_time:
    :param timeout:

    :return:
    """
    file_elements_str = ','.join(str(item) for item in file_elements)
    start_time_str = timestamp_to_str(start_time)
    logging.debug("get_phase1_detail_result start_time_str: %s, file_elements: %s, step: %s"%(start_time, file_elements, step))
    sql = "SELECT file_id, element_id, status FROM t_das_dashboard_phase1_detail WHERE (file_id, element_id) in (%s) AND stage='%s' and upt>='%s'"\
          % (file_elements_str, STEP_FIELDS_MAP[step]['stage'], start_time_str)
    #res = execute_query(sql,result_type='dict')
    res = execute_query(sql)
    while len(res) < len(file_elements) and time.time() - int(glo.get_value("time_difference", defValue=0)) - start_time < timeout:
        res = execute_query(sql)
        time.sleep(3)

    return res


def get_phase3_detail_result(files, step, start_time, timeout=60):
    """

    :param file_elements:
    :param step:
    :param start_time:
    :param timeout:
    :return:
    """

    file_ids_str = ','.join(str(item) for item in files)
    start_time_str = timestamp_to_str(start_time)
    sql = "SELECT file_id, status FROM t_das_dashboard_phase3_detail WHERE  file_id in (%s)  AND stage='%s' and upt>='%s'"\
          %(file_ids_str, STEP_FIELDS_MAP[step]['stage'],start_time_str)
    res = execute_query(sql)
    while len(res)<len(files) and time.time()-int(glo.get_value("time_difference",defValue=0)) - start_time < timeout:
        res = execute_query(sql)
        time.sleep(3)

    return res


def get_process_result(target,step, start_time, timeout=60):
    if step in [0,1,3]:
        return get_phase1_detail_result(target, step, start_time, timeout)
    elif step ==4:
        return get_std_qc_element_result(target, start_time, timeout)
    elif step in [5,6]:
        return get_phase3_detail_result(target, step, start_time, timeout)


def get_phase1_time(file_id, model_id, element_id, stage):
    """
    获取提取/结构化/标准化完成的时间

    :param file_id: 文件id
    :param model_id: 模型id
    :param element_id: 元素id
    :param stage: 阶段（Extract/Struct/StructQA/Std）

    :return: 更新时间
    """
    sql = f"SELECT upt FROM `t_das_dashboard_phase1_detail` WHERE file_id='{file_id}' AND model_id='{model_id}' AND element_id={element_id} AND stage='{stage}';"
    try:
        res = execute_query(sql)
        return res[0][0]
    except Exception as e:
        logging.error("get_phase1_time: %s" % e)

def get_phase2_time(file_id, qc_group, stage='StdQA'):
    """
    获取标准化质检完成的时间

    :param file_id: 文件id
    :param qc_group: 质检分组
    :param stage: 阶段（StdQA）

    :return: 更新时间
    """
    sql = f"SELECT upt FROM `t_das_dashboard_phase2_detail` WHERE file_id='{file_id}' AND qcgroup='{qc_group}' AND stage='{stage}';"
    try:
        res = execute_query(sql)
        return res[0][0]
    except Exception as e:
        logging.error("get_phase2_time: %s" % e)